Histogram Model for Image Objects

---

Sergei Papulin (papulin.edu@gmail.com)

Contents

• Loading Dataset
• Defining Positional Elements
• Defining Object Elements
• Creating Histogram
• Querying
• Image Retrieval
• References

Creating virtual environment

This is an optional step. You can skip it and install packages to your current environment.

python -m venv .venv/histtest
source .venv/histtest/bin/activate
pip install \
    numpy==1.19.5 \
    matplotlib==3.0.3 \
    jupyter==1.0.0 \
    pillow==5.4.1 \
    scikit-image==0.14.2 \
    pycocotools==2.0.3 \
    himpy=0.0.1

Loading packages

import os
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.image as image_utils
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon, Rectangle
%matplotlib inline

# import skimage.io as io
import skimage.draw as draw

from himpy.histogram import operations
from himpy.executor import Parser, Evaluator
from himpy.utils import E

%load_ext autoreload
%autoreload 2

import sys
sys.path.insert(0, "../")


from utils.datasets import COCOLoader

# feature extraction
from utils.feature_extraction import (
    FeatureMerger,
    PositionSetTransformer,
    filter_data,
    create_histogram,
    create_histogram_,
    extract_elements,
    extract_element_set
)

from utils.feature_extraction.coco import COCOObjectSetTransformer, load_coco_histograms

# search engine
from utils.search_engine import SearchEngine

# plot
from utils.plot.matplotlib_plot import (
    plot_position_grid,
    plot_object_ids,
    show_operation_result,
    show_retrieved_images
)

Loading Dataset

Downloading Dataset

The COCO dataset, website:

• images
• annotations

loader = COCOLoader()

# Download the datasets
coco = loader.fetch_load()

loading annotations into memory...
Done (t=0.67s)
creating index...
index created!

Image and Annotation

# Id of some image from the dataset
IMAGE_ID = 404484

# Get an image file path
image_path = loader.load_path(IMAGE_ID)

# Plot the image
I = image_utils.imread(image_path)
plt.imshow(I)
plt.title("Image")
plt.show()

Defining Positional Elements

Low-Level Elements

# Grid params: 5 splits along Y, and 5 along X
GRID = (5, 5)

# Create a position transformer
position_transformer = PositionSetTransformer(splits=GRID, element_ndim=3)

# Set an image size
position_transformer.fit(X=I, y=None)

# Build an image in which each pixel defines a position
position_image = position_transformer.transform(X=I)

# Plot the positional element along with the initial image
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("Low-level position elements")
axes[1].imshow(position_image)
axes[1] = plot_position_grid(position_transformer, axes[1])
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(I)
axes[2] = plot_position_grid(position_transformer, axes[2])
axes[2].axis("off")
plt.show()

High-Level Elements

# Create an instance of query parser
parser = Parser()

# Definition of high-level positional elements

Ep_top    = E("1+2+3+4+5+6+7+8+9+10")
Ep_bottom = E("16+17+18+19+20+21+22+23+24+25")
Ep_left   = E("1+2+6+7+11+12+16+17+21+22")
Ep_right  = E("4+5+9+10+14+15+19+20+24+25")
Ep_center = E("7+8+9+12+13+14+17+18+19")

Eps = [
    ("top", Ep_top), 
    ("bottom", Ep_bottom), 
    ("left", Ep_left), 
    ("right", Ep_right), 
    ("center", Ep_center)
]


# Sets of high-level positional elements (they will be used for the Evaluator below)

Eps_set = { name: parser.parse_set(Ep.value) for name, Ep in Eps}
Eps_set["center"]

{'12', '13', '14', '17', '18', '19', '7', '8', '9'}

# Plot a high-level element
Ep_set_ = Eps_set["center"]

elements_image = position_transformer.filter_elements(position_image, Ep_set_)
filtered_image = position_transformer.filter_data(position_image, I, Ep_set_)

fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("High-level position elements")
axes[1].imshow(elements_image)
axes[1] = plot_position_grid(position_transformer, axes[1], Ep_set_)
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(filtered_image)
axes[2] = plot_position_grid(position_transformer, axes[2], Ep_set_)
axes[2].get_xaxis().set_visible(False)
axes[2].get_yaxis().set_visible(False)

plt.show()

Defining Object Elements

Low-Level Elements

object_names = loader.load_object_names()
print("Total number of categories: {}".format(len(object_names)))
object_names[:5]

Total number of categories: 80

[(1, 'person'), (2, 'bicycle'), (3, 'car'), (4, 'motorcycle'), (5, 'airplane')]

# Create object transformer
object_transformer = COCOObjectSetTransformer(coco)

# Build object mask
object_image = object_transformer.fit_transform(X=None, y=None, ids=IMAGE_ID)

# Mask image based on segments
image__object_filtered = object_transformer.filter_data(object_image, I)

# Plot initial image, object image, and filtered image
fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("Low-level object elements")
axes[1].imshow(object_image)
axes[1] = plot_object_ids(object_image, axes[1])
# axes[1] = utils.plot_object_edges(object_transformer, axes[1])
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(I)
axes[2].imshow(image__object_filtered, alpha=0.5)
axes[2] = plot_object_ids(object_image, axes[2])
# axes[2] = utils.plot_object_edges(object_transformer, axes[2])
axes[2].axis("off")

plt.show()

object_transformer.get_element_name(extract_elements(object_image))

[('person', 1),
 ('dog', 18),
 ('potted plant', 64),
 ('tv', 72),
 ('teddy bear', 88)]

High-Level Elements

# Definition of high-level positional elements

Eo_living_beings = E("1+18")
Eo_person        = E("1")
Eo_dog           = E("18")


Eos = [
    ("living_beings", Eo_living_beings),
    ("person", Eo_person),
    ("dog", Eo_dog),
]


# Sets of high-level positional elements (they will be used for the Evaluator below)

Eos_set = { name: parser.parse_set(Eo.value) for name, Eo in Eos}
Eos_set["living_beings"]

{'1', '18'}

# Plot a high-level element
Eo_set_ = Eos_set["living_beings"]

elements_image = object_transformer.filter_elements(object_image, Eo_set_)
filtered_image = object_transformer.filter_data(object_image, I, Eo_set_)

fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].axis("off")
axes[1].set_title("High-level object elements")
axes[1].imshow(elements_image)
axes[1] = plot_object_ids(elements_image, axes[1], Eo_set_)
# axes[1] = utils.plot_object_edges(object_transformer, axes[1], Eo_set_)
axes[1].axis("off")
axes[2].set_title("Matching image and elements")
axes[2].imshow(I)
axes[2].imshow(filtered_image, alpha=0.5)
axes[2] = plot_object_ids(elements_image, axes[2], Eo_set_)
# axes[2] = utils.plot_object_edges(object_transformer, axes[2], Eo_set_)
axes[2].axis("off")

plt.show()

object_transformer.get_element_name(Eos_set["living_beings"])

[('dog', '18'), ('person', '1')]

Creating Histogram

# Option 1
hist = create_histogram((position_image, object_image))
hist.to_dict()

{('1', '72'): 9.114583333333334e-05,
 ('3', '1'): 0.0037239583333333335,
 ('4', '1'): 0.0130078125,
 ('6', '72'): 0.010208333333333333,
 ('7', '18'): 0.0006119791666666667,
 ('8', '1'): 0.0004166666666666667,
 ('9', '1'): 0.032578125,
 ('10', '1'): 0.0019921875,
 ('10', '64'): 0.007005208333333333,
 ('11', '72'): 0.003203125,
 ('11', '88'): 0.0012109375,
 ('12', '18'): 0.016315104166666667,
 ('12', '88'): 0.0055859375,
 ('13', '18'): 0.013619791666666667,
 ('14', '1'): 0.0036067708333333334,
 ('14', '64'): 0.016848958333333334,
 ('15', '64'): 0.02421875,
 ('17', '18'): 0.001953125,
 ('17', '88'): 0.0003515625,
 ('18', '18'): 0.0014713541666666666,
 ('19', '64'): 0.0004427083333333333,
 ('20', '64'): 0.0014192708333333334}

# Option 2.a Merge features into a single image
feature_merger = FeatureMerger()
merged_image = feature_merger.fit_transform((position_image, object_image))
merged_image

rec.array([[( 1, 0), ( 1, 0), ( 1, 0), ..., ( 5, 0), ( 5, 0), ( 5, 0)],
           [( 1, 0), ( 1, 0), ( 1, 0), ..., ( 5, 0), ( 5, 0), ( 5, 0)],
           [( 1, 0), ( 1, 0), ( 1, 0), ..., ( 5, 0), ( 5, 0), ( 5, 0)],
           ...,
           [(21, 0), (21, 0), (21, 0), ..., (25, 0), (25, 0), (25, 0)],
           [(21, 0), (21, 0), (21, 0), ..., (25, 0), (25, 0), (25, 0)],
           [(21, 0), (21, 0), (21, 0), ..., (25, 0), (25, 0), (25, 0)]],
          dtype=[('f0', 'u1'), ('f1', '<i8')])

# Option 2.b Create a histogram
hist = create_histogram_(merged_image)
hist.to_dict()

{('1', '72'): 9.114583333333334e-05,
 ('3', '1'): 0.0037239583333333335,
 ('4', '1'): 0.0130078125,
 ('6', '72'): 0.010208333333333333,
 ('7', '18'): 0.0006119791666666667,
 ('8', '1'): 0.0004166666666666667,
 ('9', '1'): 0.032578125,
 ('10', '1'): 0.0019921875,
 ('10', '64'): 0.007005208333333333,
 ('11', '72'): 0.003203125,
 ('11', '88'): 0.0012109375,
 ('12', '18'): 0.016315104166666667,
 ('12', '88'): 0.0055859375,
 ('13', '18'): 0.013619791666666667,
 ('14', '1'): 0.0036067708333333334,
 ('14', '64'): 0.016848958333333334,
 ('15', '64'): 0.02421875,
 ('17', '18'): 0.001953125,
 ('17', '88'): 0.0003515625,
 ('18', '18'): 0.0014713541666666666,
 ('19', '64'): 0.0004427083333333333,
 ('20', '64'): 0.0014192708333333334}

Querying

# TODO: 
# OBJ_COLOR = {str(el[0]): np.random.randint(0, 255, 3) for el in object_names}

high_level_elements = {
    0: Eps_set, # positions
    1: Eos_set  # objects
}

evaluator = Evaluator(operations, hist, high_level_elements=high_level_elements)

POS1 = "center"
OBJ1 = "person"

POS2 = "left"
OBJ2 = "dog"

E1 = E(POS1, OBJ1)
E2 = E(POS2, OBJ2)

E1_expr = parser.parse_string(E1.value)
HE1 = evaluator.eval(E1_expr)
print("Expression for E1:\n{}".format(E1.value))
print("\nThe parsed expressino for E1 in the postfix notation:\n{}".format(E1_expr))
print("\nHistogram of E1 given the image:\n{}".format(HE1.to_dict()))
print("\nValue of presence for E1:\n{}".format(HE1.sum()))

Expression for E1:
(center,person)

The parsed expressino for E1 in the postfix notation:
[('center', 'person')]

Histogram of E1 given the image:
{('8', '1'): 0.0004166666666666667, ('14', '1'): 0.0036067708333333334, ('9', '1'): 0.032578125}

Value of presence for E1:
0.0366015625

E2_expr = parser.parse_string(E2.value)
HE2 = evaluator.eval(E2_expr)
print("Expression for E2:\n{}".format(E2.value))
print("\nThe parsed expressino for E2 in the postfix notation:\n{}".format(E2_expr))
print("\nHistogram of E2 given the image:\n{}".format(HE2.to_dict()))
print("\nValue of presence for E2:\n{}".format(HE2.sum()))

Expression for E2:
(left,dog)

The parsed expressino for E2 in the postfix notation:
[('left', 'dog')]

Histogram of E2 given the image:
{('7', '18'): 0.0006119791666666667, ('17', '18'): 0.001953125, ('12', '18'): 0.016315104166666667}

Value of presence for E2:
0.018880208333333336

# Plot histogram elements

E1_set = extract_element_set(HE1, 2)
E2_set = extract_element_set(HE2, 2)

E1_image = filter_data(I, merged_image, HE1.elements())
E2_image = filter_data(I, merged_image, HE2.elements())

fig, axes = plt.subplots(1, 3, figsize=(14,20))
axes[0].set_title("Initial Image")
axes[0].imshow(I)
axes[0].get_xaxis().set_visible(False)
axes[0].get_yaxis().set_visible(False)
axes[1].set_title("E1: {}".format(E1.value))
axes[1].imshow(I)
axes[1].imshow(E1_image, alpha=0.8)
axes[1] = plot_position_grid(position_transformer, axes[1], E1_set[0])
axes[1] = plot_object_ids(elements_image, axes[1],  E1_set[1])
axes[1].get_xaxis().set_visible(False)
axes[1].get_yaxis().set_visible(False)
axes[2].set_title("E2: {}".format(E2.value))
axes[2].imshow(I)
axes[2].imshow(E2_image, alpha=0.8)
axes[2] = plot_position_grid(position_transformer, axes[2], E2_set[0])
axes[2] = plot_object_ids(elements_image, axes[2],  E2_set[1])
axes[2].get_xaxis().set_visible(False)
axes[2].get_yaxis().set_visible(False)

plt.show()

Operations on Histogram Elements

Example for Union

# Expression with union
E_union = E1 + E2

# Parsed expression
E_union_expr = parser.parse_string(E_union.value)

# Calculate histogram value
HE_union = evaluator.eval(E_union_expr)

print("Expression for E_union:\n{}".format(E_union))
print("\nThe parsed expression for E_union in the postfix notation:\n{}".format(E_union_expr))
print("\nHistogram of E_union given the image:\n{}".format(HE_union.to_dict()))
print("\nValue of presence for E_union:\n{}".format(HE_union.sum()))

Expression for E_union:
((center,person)+(left,dog))

The parsed expression for E_union in the postfix notation:
[('center', 'person'), ('left', 'dog'), '+']

Histogram of E_union given the image:
{('8', '1'): 0.0004166666666666667, ('14', '1'): 0.0036067708333333334, ('7', '18'): 0.0006119791666666667, ('17', '18'): 0.001953125, ('9', '1'): 0.032578125, ('12', '18'): 0.016315104166666667}

Value of presence for E_union:
0.05548177083333333

# Extract ids of non-zero elements for each feature
E_result_set = extract_element_set(HE_union, 2)

# Plot elements and result
transformers = (position_transformer, object_transformer)
titles = ["E1: {}".format(E1), "E2: {}".format(E2), "Result: {}".format(E_union)]
show_operation_result(I, merged_image, "f1", HE1, HE2, HE_union, transformers, titles)

Other operations

operation_list = [
    # set operations
    ("union",          "+",    E1 + E2), 
    ("intersection",   "*",    E1 * E2),
    ("substraction",   "-",    E1 - E2),  # or exception, or E1.Sub(E2)
    # logic operations
    ("and",            "&",    E1 & E2),  # or E1.And(E2)
    ("or",             "|",    E1 | E2),  # or E1.Or(E2)
    ("xor",            "^",    E1 ^ E2),  # or E1.Xor(E2)
    ("xsubstraction",  "Xsub", E1.Xsub(E2)),
]

for op_name, op_sign, op in operation_list:
    E_expr = parser.parse_string(op.value)
    HE = evaluator.eval(E_expr)
    print("{:12}{:^12}{:10}".format("Operation", "Sign", "Result"))
    print("{}".format("-"*34))
    print("{:12}{:^12}{:.5f}".format(op_name, op_sign, HE.sum()))
    E_result_set = extract_element_set(HE, 2)
    titles = ["E1: {}".format(E1), "E2: {}".format(E2), "Result: {}".format(op)]
    show_operation_result(I, merged_image, "f1", HE1, HE2, HE, transformers, titles)

Operation       Sign    Result    
----------------------------------
union            +      0.05548

Operation       Sign    Result    
----------------------------------
intersection     *      0.00000

Operation       Sign    Result    
----------------------------------
substraction     -      0.03660

Operation       Sign    Result    
----------------------------------
and              &      0.01888

Operation       Sign    Result    
----------------------------------
or               |      0.05548

Operation       Sign    Result    
----------------------------------
xor              ^      0.03660

Operation       Sign    Result    
----------------------------------
xsubstraction    Xsub    0.00000

Image Retrieval

Expression as query

# Load histograms of coco images if exist, otherwise transform images to histograms and return
hists = load_coco_histograms(coco)
hists[:5]

[(397133, <himpy.histogram.Histogram1D at 0x7f4b941cb3c8>),
 (37777, <himpy.histogram.Histogram1D at 0x7f4b941d3630>),
 (252219, <himpy.histogram.Histogram1D at 0x7f4b9427fcc0>),
 (87038, <himpy.histogram.Histogram1D at 0x7f4b9427fd68>),
 (174482, <himpy.histogram.Histogram1D at 0x7f4b9427f320>)]

# Initialize a search engine
search_engine = SearchEngine(hists, parser, evaluator)

TOP_N = 20

# Define your query
query = E("left", "dog") & E("center", "person")

# Retrieve images using the query
ranked_images = search_engine.retrieve(query, topN=TOP_N)
print("Total retrieved images:", len(ranked_images))
ranked_images[:5]

Total retrieved images: 20

[(432553, 0.10005122950819673),
 (236166, 0.09953418427230046),
 (107226, 0.08212565104166666),
 (246454, 0.07708740234375),
 (117525, 0.072228)]

# Compose paths of retrieved images
image_paths = [
    loader.load_path(image_id)
    for image_id, _ in ranked_images
]
image_paths[:1]

['datasets/coco/val2017/000000432553.jpg']

# Show top ranked images
show_retrieved_images(ranked_images, image_paths, limit=TOP_N, title="Query: {}".format(query.value))

Image sample as query

# Take a sample image and its histogram from the list of histograms that were created previously
sample_image_id = hists[4][0]
sample_hist = hists[4][1]

# Show the sample image
sample_image_path = loader.load_path(sample_image_id)
I = image_utils.imread(sample_image_path)
plt.imshow(I)
plt.title("Image")
plt.show()

# Retrieve images similar to the sample
ranked_images__sample = search_engine.retrieve(sample_hist, topN=TOP_N)
print("Total retrieved images:", len(ranked_images__sample))
ranked_images__sample[:5]

Total retrieved images: 20

[(174482, 0.36078849871134017),
 (109055, 0.2703729730455326),
 (492937, 0.25539785193227954),
 (274687, 0.19363381926546394),
 (101762, 0.18673747583762884)]

# Compose paths of retrieved images
image_paths__sample = [
    loader.load_path(image_id) 
    for image_id, _ in ranked_images__sample
]
# Show top ranked images
show_retrieved_images(
    ranked_images__sample, 
    image_paths__sample, 
    limit=TOP_N,
    title="Query: {}".format("Sample Image")
)

References

• COCO (Dataset): Common Objects in Context
• Papulin S. Introduction to Histogram Model
• Papulin S. Multidimensional Histogram Model